{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logs Analytics Dashboard\n",
    "This dashboard can be used for performing visual analytics on the server log files. You can interact with any chart in the dashboard, like so:\n",
    "\n",
    "1. Click on a bar in any bar chart to filter the whole dataset by that bar's value\n",
    "2. Click on a slice of any pie chart to filter the whole dataset by that pie slices value\n",
    "\n",
    "Fun things to try:\n",
    "1. Select any specific day in the `Daily Events` bar chart (for e.g. why do first and last days have fewer events?)\n",
    "2. Find out which product queries resulted in server error (click on `Server Error` slice of `Events By Status` pie chart)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import ipywidgets as widgets\n",
    "import bqplot as bq\n",
    "import bqplot.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_status_code(x):\n",
    "    \"\"\"\n",
    "    map integer http status to  string status code\n",
    "    \"\"\"\n",
    "    if x >= 200 and x < 300:\n",
    "        return 'SUCCESS'\n",
    "    elif x >= 300 and x < 400:\n",
    "        return 'REDIRECT'\n",
    "    elif x >= 400 and x < 500:\n",
    "        return 'CLIENT ERROR'\n",
    "    elif x >= 500:\n",
    "        return 'SERVER ERROR'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_events_by_hour(log_data_slice):\n",
    "    \"\"\"\n",
    "    get event counts by hour\n",
    "    \"\"\"\n",
    "    return log_data_slice['status_code']\\\n",
    "    .groupby(lambda x: x.hour)\\\n",
    "    .count()\\\n",
    "    .reindex(np.arange(24))\\\n",
    "    .fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load data into pandas and do some munging\n",
    "log_data = pd.read_csv('access.log', sep=' ', header=None)\n",
    "log_data.drop([1, 2], axis=1, inplace=True)\n",
    "log_data.columns = ['ip_address', 'timestamp', 'request', 'status', 'col1', 'url', 'agent', 'col2']\n",
    "\n",
    "log_data.index = pd.to_datetime(log_data['timestamp'].str.replace('[\\[\\]]', ''), \n",
    "                                format='%d/%b/%Y:%H:%M:%S')\n",
    "log_data.drop('timestamp', axis=1, inplace=True)\n",
    "\n",
    "# add extra columns for easy querying\n",
    "log_data['status_code'] = log_data['status'].map(get_status_code)\n",
    "log_data['category'] = log_data['request'].str.extract('categoryId=(.*)&', expand=False)\n",
    "log_data['product'] = log_data['request'].str.extract('productId=(.*)&', expand=False)\n",
    "log_data['hour'] = log_data.index.map(lambda x: x.hour)\n",
    "log_data['day'] = log_data.index.strftime('%Y-%m-%d')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_days = log_data['day'].value_counts().index\n",
    "all_hours = np.arange(24)\n",
    "all_categories = list(log_data['category'].value_counts().index)\n",
    "all_products = list(log_data['product'].value_counts().index)\n",
    "all_status_codes = list(log_data['status_code'].value_counts().index)\n",
    "\n",
    "category_colors = dict(zip(all_categories, bq.CATEGORY10))\n",
    "status_label_colors = dict([('SUCCESS', '#006d2c'), \n",
    "                            ('CLIENT ERROR', '#fc8d59'), \n",
    "                            ('SERVER ERROR', '#a63603')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# daily events bar chart\n",
    "daily_events_fig = plt.figure(title='Daily Events', \n",
    "                              animation_duration=1000,\n",
    "                              layout=widgets.Layout(width='900px', height='500px'))\n",
    "\n",
    "plt.scales(scales={'x': bq.DateScale()})\n",
    "\n",
    "# intsel = BrushIntervalSelector(scale=, marks=[hist])\n",
    "bar_axes_options = {'x': {'grid_lines': 'none'},\n",
    "                    'y': {'tick_format': ','}}\n",
    "common_bar_options = dict(interactions={'click': 'select'},\n",
    "                          selected_style={'stroke': 'Red',\n",
    "                                          'stroke-width': 4},\n",
    "                          axes_options=bar_axes_options)\n",
    "daily_events_bar = plt.bar(pd.to_datetime(all_days), [],\n",
    "                           colors=['dodgerblue'], \n",
    "                           opacities=[.8] * len(all_days),\n",
    "                           **common_bar_options)\n",
    "\n",
    "num_days = len(all_days)\n",
    "filtered_daily_events_bar = plt.bar(pd.to_datetime(all_days),\n",
    "                                    np.zeros(num_days),\n",
    "                                    colors=['lightgreen'], \n",
    "                                    opacities=[.8] * num_days,\n",
    "                                    axes_options=bar_axes_options)\n",
    "\n",
    "# hourly events line chart\n",
    "hourly_events_fig = plt.figure(title='Hourly Events',\n",
    "                               animation_duration=1000,\n",
    "                               layout=widgets.Layout(width='600px', height='500px'))\n",
    "\n",
    "plt.scales(scales={'x': bq.OrdinalScale()})\n",
    "hourly_events_bar = plt.bar(all_hours,[],\n",
    "                            colors=['goldenrod'],\n",
    "                            opacities=[.8] * 24,\n",
    "                            padding=.2,\n",
    "                            **common_bar_options)\n",
    "\n",
    "filtered_hourly_events_bar = plt.bar(all_hours, [],\n",
    "                                     colors=['lightgreen'],\n",
    "                                     opacities=[.8] * 24,\n",
    "                                     padding=.2,\n",
    "                                     axes_options=bar_axes_options)\n",
    "\n",
    "products_fig = plt.figure(title='Events By Product', \n",
    "                          animation_duration=1000,\n",
    "                          fig_margin=dict(top=60, bottom=20, left=100, right=40),\n",
    "                          layout=widgets.Layout(width='550px', height='500px'))\n",
    "products_bar = plt.bar(all_products, [],\n",
    "                       colors=['salmon'],\n",
    "                       orientation='horizontal',\n",
    "                       opacities=[.8] * len(all_products),\n",
    "                       padding=.2,\n",
    "                       **common_bar_options)\n",
    "\n",
    "common_pie_args = dict(display_labels='outside',\n",
    "                       interactions={'click': 'select'},\n",
    "                       selected_style={'stroke': 'white',\n",
    "                                       'stroke-width': 3},\n",
    "                       inner_radius=80,\n",
    "                       apply_clip=False)\n",
    "# categories pie chart\n",
    "categories_fig = plt.figure(title='Events By Category', animation_duration=1000,\n",
    "                           layout=widgets.Layout(width='550px', height='500px'))\n",
    "categories_pie = plt.pie([], labels=all_categories, **common_pie_args)\n",
    "\n",
    "# status codes pie chart\n",
    "status_codes_fig = plt.figure(title='Events By Status', animation_duration=1000,\n",
    "                              layout=widgets.Layout(width='550px', height='500px'))\n",
    "status_codes_pie = plt.pie([], labels=all_status_codes, **common_pie_args)\n",
    "\n",
    "# buttons for updating and resetting filters\n",
    "update_btn = widgets.Button(description='Update', button_style='Success')\n",
    "reset_btn = widgets.Button(description='Reset', button_style='Success')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fields = ['day', 'hour', 'product', 'category', 'status_code']\n",
    "field_vals = [all_days, all_hours, all_products, all_categories, all_status_codes]\n",
    "plots = [daily_events_bar, hourly_events_bar, products_bar, categories_pie, status_codes_pie]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_filters():\n",
    "    filters_dict = {}\n",
    "    for field, field_val, plot in zip(fields, field_vals, plots):\n",
    "        if plot.selected:\n",
    "            selected_vals = [field_val[i] for i in plot.selected]\n",
    "            filters_dict[field] = selected_vals\n",
    "    return filters_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def apply_filters(df, filters):\n",
    "    filtered_df = df\n",
    "    \n",
    "    for k, v in filters.items():\n",
    "        if v and len(v) > 0:\n",
    "            filtered_df = filtered_df[filtered_df[k].isin(v)]\n",
    "    return filtered_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_plots(*args):\n",
    "    global filters, filtered_log_data    \n",
    "    filters = generate_filters()    \n",
    "    filtered_log_data = apply_filters(log_data, filters)\n",
    "    \n",
    "    daily_events = filtered_log_data\\\n",
    "        .resample('D')\\\n",
    "        .count()['status_code']\\\n",
    "        .reindex(pd.to_datetime(all_days))\\\n",
    "        .fillna(0)\n",
    "    daily_events_bar.y = daily_events\n",
    "    \n",
    "    hourly_events_bar.y = get_events_by_hour(filtered_log_data)        \n",
    "    \n",
    "    products_bar.y = filtered_log_data['product']\\\n",
    "        .value_counts()\\\n",
    "        .reindex(all_products)\\\n",
    "        .fillna(0)\n",
    "    \n",
    "    events_by_category = filtered_log_data['category'].value_counts()\n",
    "        \n",
    "    with categories_pie.hold_sync():\n",
    "        categories_pie.labels = list(events_by_category.index)\n",
    "        categories_pie.sizes = events_by_category\n",
    "        categories_pie.colors = [category_colors[d] for d in categories_pie.labels]\n",
    "    \n",
    "    events_by_status_code = filtered_log_data['status_code'].value_counts()\n",
    "        \n",
    "    with status_codes_pie.hold_sync():\n",
    "        status_codes_pie.labels = list(events_by_status_code.index)\n",
    "        status_codes_pie.sizes = events_by_status_code.values\n",
    "        status_codes_pie.colors = [status_label_colors[d] for d in status_codes_pie.labels]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reset_filters(*args):\n",
    "    for plot in plots:\n",
    "        plot.selected = None\n",
    "    update_plots(None)\n",
    "        \n",
    "update_btn.on_click(lambda btn: update_plots(None))\n",
    "reset_btn.on_click(lambda btn: reset_filters())\n",
    "btns_layout = widgets.VBox([update_btn, reset_btn], layout=widgets.Layout(overflow_x='hidden'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "update_plots(None)\n",
    "\n",
    "parent_plots = widgets.HBox([daily_events_fig, hourly_events_fig])\n",
    "child_plots = widgets.HBox([products_fig, categories_fig, status_codes_fig])\n",
    "widgets.VBox([widgets.HBox([parent_plots, btns_layout]),child_plots])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
